#!/bin/bash
OK=0
WARN=1
CRITICAL=2
UNKNOWN=3
PATH="/bin:/sbin:/usr/bin:/usr/sbin"
version=13
Ok_name=''
Fail_name=''
Judge=0
EFile="/tmp/eos_check_log.err"
Ver='v1.1'
currentcpuload=`uptime | awk -F"load average:" '{print $NF}'|awk -F" " '{print $1}'|awk -F"." '{print $1}'`
if [ $currentcpuload -ge 100 ];then
    echo "cpuload is $currentcpuload"
    exit $UNKNOWN
fi

if ! command -v mail > /dev/null; then
	echo "No mail installed"
	/cvmfs/common.ihep.ac.cn/cc/Python/Python-2.6/bin/python /cvmfs/common.ihep.ac.cn/cc/sysgroup/smtp.pyc -s `hostname -s`" has no mail installed" -c biyujiang@ihep.ac.cn -t "no mail installed" lihaibo@ihep.ac.cn >> /dev/null
	exit 130
fi

function send_mail(){
	echo "$2" | mail -s "`hostname -s` eosclient $1" -c biyujiang@ihep.ac.cn lihaibo@ihep.ac.cn >/dev/null
#	/cvmfs/common.ihep.ac.cn/cc/Python/Python-2.6/bin/python /cvmfs/common.ihep.ac.cn/cc/sysgroup/smtp.pyc -s `hostname -s`" eosclient $1" -c biyujiang@ihep.ac.cn -t "$2" lihaibo@ihep.ac.cn >> /dev/null
#/afs/ihep.ac.cn/soft/common/Python/Python-2.6/bin/python /afs/ihep.ac.cn/soft/common/sysgroup/smtp.pyc -s `hostname -s`" eosclient $1" -c biyujiang@ihep.ac.cn -t "$2" lihaibo@ihep.ac.cn >> /dev/null
}

function check_eosd() {
if [ "$choice" == "/mnt/hxmt/home/nagios" ];then
	TempE=`ps aux|grep -v grep|grep "/usr/bin/eosd /mnt/hxmt"`
	if [ "t$TempE" == "t" ];then
		cd /etc/init.d
		./eosd stop hxmt
		umount -l /mnt/hxmt
		rm -rf /etc/mtab~*
		./eosd start hxmt
		send_mail "warnning:/mnt/hxmt started" "/mnt/hxmt not running,now started"
		continue
	fi

	/usr/bin/timeout 15 strace test -w $choice/ss 1>/dev/null 2>$EFile
	if [ $? != 0 ]
	then
		ETemp=`grep "Transport endpoint is not connected" $EFile`
		if [[ ! -n "$ETemp" || "t$ETemp" != "t" ]];then
			umount -l /mnt/hxmt
			umount -l /mnt/hxmt
			service eosd restart hxmt
			send_mail "Tranport Error happen for hxmt" "restart eosd"
			continue
		fi

		Temp=`/bin/mount -l|grep "eoshxmt"`
		if [ "t$Temp" != "t" ];then
			send_mail "test reading /mnt/hxmt locked" "/eos still alive"
			continue
		fi

		TempSP=`ps aux|grep -v grep|grep "./eosd stop"`
		if [ "t$TempSP" != "t" ];then
			send_mail "stop /mnt/hxmt locked" "/mnt/hxmt can not stop"
			continue
		fi

		TempST=`ps aux|grep -v grep|grep "./eosd start"`
		if [ "t$TempST" != "t" ];then
			send_mail "start /mnt/hxmt locked" "/mnt/hxmt can not start"
			continue
		fi

		cd /etc/init.d
		./eosd stop hxmt
		./eosd start hxmt
	else
		Ok_name="${Ok_name} ${choice}"
		echo "$Ok_name are OK "
	fi
fi

}

for choice in  "/mnt/hxmt/home/nagios"
do
	check_eosd "$choice"
done


#add by lihaibo, used for checking hxmt eosd status on login nodes 
#add by lihaibo, 2019-09-11

#add by lihaibo, update the python to /cvmfs,2019-12-03
